# This is a config file for BeeGFS storage nodes.
# http://www.beegfs.com


# --- [Table of Contents] ---
#
# 1) Settings
# 2) Command Line Arguments
# 3) Basic Settings Documentation
# 4) Advanced Settings Documentation


#
# --- Section 1.1: [Basic Settings] ---
#

sysMgmtdHost                 = {{ beegfs_mgmtd_host }}

storeStorageDirectory        = {{ beegfs_storage_dir }}
storeAllowFirstRunInit       = true


#
# --- Section 1.2: [Advanced Settings] ---
#

connAuthFile                 =
connBacklogTCP               = 1024
connInterfacesFile           =
connMaxInternodeNum          = 64

connMgmtdPortTCP             = 8008
connMgmtdPortUDP             = 8008
connStoragePortTCP           = 8003
connStoragePortUDP           = 8003
connPortShift                = 0

connNetFilterFile            =

connUseRDMA                  = true
connRDMATypeOfService        = 0
connTcpOnlyFilterFile        =

logLevel                     = 3
logNoDate                    = false
logNumLines                  = 50000
logNumRotatedFiles           = 5
logStdFile                   = /var/log/beegfs-storage.log

runDaemonized                = true

sysResyncSafetyThresholdMins = 10
sysTargetOfflineTimeoutSecs  = 180

tuneBindToNumaZone           = 0
tuneFileReadAheadSize        = 2m
tuneFileReadAheadTriggerSize = 256k
tuneFileReadSize             = 128k
tuneFileWriteSize            = 128k
tuneFileWriteSyncSize        = 0m

tuneNumResyncGatherSlaves    = 6
tuneNumResyncSlaves          = 12
tuneNumStreamListeners       = 4
tuneNumWorkers               = 24
tuneUseAggressiveStreamPoll  = false
tuneUsePerTargetWorkers      = true
tuneUsePerUserMsgQueues      = true
tuneWorkerBufSize            = 8m


#
# --- Section 2: [Command Line Arguments] ---
#

# Use the command line argument "cfgFile=/etc/anotherconfig.conf" to
# specify a different config file for beegfs_storage.
#
# All other options in this file can also be used as command line
# arguments, overriding the corresponding config file values.


#
# --- Section 3: [Basic Settings Documentation] ---
#

# [sysMgmtdHost]
# Hostname (or IP) of the host running the management service.
# (See also "connMgmtdPortUDP" and "connMgmtdPortTCP")
# Default: <none>

# [storeStorageDirectory]
# The absoute path to a storage target. A storage target is a directory where
# the file system can store raw user file contents. Multiple targets can be
# specified as a comma-separated list.
# Example: /mnt/beegfs_storage1,/mnt/beegfs_storage2
# Default: <none>

# [storeAllowFirstRunInit]
# Enables or disables daemon startup with an uninitialized storage directory.
# This can be used to make sure that the daemon does not run when the storage
# partition is not mounted (e.g. because it needs repair after a power outage).
# Note: This setting must be enabled during first startup of the daemon, but
#    may be disabled afterwards.
# Default: true


#
# --- Section 4: [Advanced Settings Documentation] ---
#

#
# --- Section 4.1: [Connections & Communication] ---
#

# [connAuthFile]
# The path to a file that contains a shared secret for connection based
# authentication. Only peers that use the same shared secret will be able to
# connect.
# Default: <none>

# [connBacklogTCP]
# The TCP listen backlog.
# Default: 128

# [connInterfacesFile]
# The path to a text file that specifies the names of the interfaces, which
# may be used for communication. One interface per line. The line number also
# defines the priority of the interface.
# Example: "ib0" in the first line, "eth0" in the second line.
# Values: This setting is optional. If unspecified, all available interfaces
#    will be used and priorities will be assigned automatically.
# Note: This has no influence on outgoing connections. The information is sent
#    to other hosts to inform them about possible communication paths.
# Default: <none>

# [connMaxInternodeNum]
# The maximum number of simultaneous connections to the same node.
# Default: 12

# [connMgmtdPortUDP], [connMgmtdPortTCP]
# The UDP and TCP ports of the management node.
# Default: 8008

# [connStoragePortUDP], [connStoragePortTCP]
# The UDP and TCP ports of the storage node.
# Default: 8003

# [connPortShift]
# Shifts all following UDP and TCP ports according to the specified value.
# Intended to make port configuration easier in case you do not want to
# configure each port individually.
# Default: 0

# [connNetFilterFile]
# The path to a text file that specifies allowed IP subnets, which may be used
# for outgoing communication. One subnet per line in classless notation (IP
# address and number of significant bits).
# Example: "192.168.10.0/24" in the first line, "192.168.20.0/24" in the second
#    line.
# Values: This setting is optional. If unspecified, all addresses are allowed
#    for outgoing communication.
# Default: <none>

# [connUseRDMA]
# Enables the use of Remote Direct Memory Access (RDMA) for Infiniband.
# For this setting to be effective, OFED ibverbs support has to be enabled at
# compile time of the beegfs_opentk library.
# Default: true

# [connRDMATypeOfService]
# Infiniband provides the option to set a type of service for an application.
# This type of service can be used by your subnet manager to provide Quality of
# Service functionality (e.g. setting different service levels).
# In openSM the service type will be mapped to the parameter qos-class, which
# can be handled in your QoS configuration.
# See
# www.openfabrics.org/downloads/OFED/ofed-1.4/OFED-1.4-docs/QoS_management_in_OpenSM.txt
# for more information on how to configure openSM for QoS.
# This parameter sets the type of service for all outgoing connections of this
# daemon.
# Default: 0 (Max: 255)

# [connTcpOnlyFilterFile]
# The path to a text file that specifies IP address ranges to which no RDMA connection should be
# established. This is useful e.g. for environments where all hosts support RDMA, but some hosts
# cannot connect via RDMA to some other hosts.
# Example: "192.168.10.0/24" in the first line, "192.168.20.0/24" in the second
#    line.
# Values: This setting is optional.
# Default: <none>

#
# --- Section 4.2: [Logging] ---
#

# [logLevel]
# Defines the amount of output messages. The higher this level, the more
# detailed the log messages will be.
# Note: Levels above 3 might decrease performance.
# Default: 3 (Max: 5)

# [logNoDate]
# Defines whether "date & time" (=false) or the current "time only" (=true)
# should be logged.
# Default: false

# [logNumLines]
# The maximum number of lines per log file.
# Default: 50000

# [logNumRotatedFiles]
# The number of old files to keep when "logNumLines" is reached and the log file
# is rewritten (log rotation).
# Default: 5

# [logStdFile]
# The path and filename of the log file for standard log messages. If no name
# is specified, the messages will be written to the console.
# Default: /var/log/beegfs_storage.log


#
# --- Section 4.4: [Startup] ---
#

# [runDaemonized]
# Detach the process from its parent (and from stdin/-out/-err).
# Default: true


#
# --- Section 4.5: [System Settings] ---
#

# [sysResyncSafetyThresholdMins]
# Automatic mirror resyncs use the last successful communication time between
# two mirror buddies to skip verification of files that were not recently
# modified before a server went offline. As BeeGFS uses server-side write
# caching (where the cache is flushed to disk every minute by the Linux kernel),
# it is possible that a server looses its write cache in case of a crash, which
# contained data before the last successful communication. This value adds an
# extra amount of time to the last successful communication timestamp to include
# the time window of a potential cache loss.
# The value may be 0 (which doesn't mean there is no threshold) to completely
# disable the use of the last successful communication timestamp,
# i.e. that a full resync will be done.
# Values: time in minutes
# Default: 10

# [sysTargetOfflineTimeoutSecs]
# Timeout until targets on a storage server are considered offline by the
# management node when no target state updates can be fetched from that server.
# Note: This must be the same value as in the /etc/beegfs/beegfs-mgmtd.conf on
# the management node.
# Values: time in seconds
# Default: 180


#
# --- Section 4.6: [Tuning] ---
#

# [tuneBindToNumaZone]
# Defines the zero-based NUMA zone number to which all threads of this process
# should be bound. If unset, all available CPU cores may be used.
# Zone binding is especially useful if the corresponding devices (e.g. storage
# controller and network card) are also attached to the same zone.
# Note: The Linux kernel shows NUMA zones at /sys/devices/system/node/nodeXY
# Default: <unset>

# [tuneFileReadAheadSize], [tuneFileReadAheadTriggerSize]
# tuneFileReadAheadSize is the byte range submitted to the kernel for read-head
# after at least tuneFileReadAheadTriggerSize file bytes were read sequentially
# from a target.
# Values: A typical setting is tuneFileReadAheadSize=2m. The optimal setting
#    depends on your storage system configuration (e.g. your RAID layout).
# Default: tuneFileReadAheadSize=0, tuneFileReadAheadTriggerSize=4m

# [tuneFileReadSize], [tuneFileWriteSize]
# The maximum amount of data that the server should write to (or read from)
# the underlying local file system in a single operation.
# Note: Setting these values higher than the file chunk size or
#    tuneWorkerBufSize has no effect.
# Default: tuneFileReadSize=128k, tuneFileWriteSize=128k

# [tuneFileWriteSyncSize]
# The number of sequentially written bytes (per file) after which the kernel
# will be advised to commit the written data to the underlying storage device.
# This is intended to avoid delays until the kernel notices that it is time to
# commit written data, which would reduce streaming write throughput.
# Note: When this setting is enabled, it is important to use the deadline
#    scheduler (/sys/block/<...>/scheduler) to avoid reader starvation. It is
#    also important to use a large request queue (/sys/block/<...>/nr_requests),
#    as writes can only be asynchronous while there are free slots in the queue.
# Values: "0" disables this mechanism. Use "32m" (or a close even multiple of
#    your RAID stripe set size) to test the effects of this.
# Default: 0

# [tuneNumResyncGatherSlaves]
# The number of threads (per target) used to gather file system information for
# a buddy mirror resync.
# Default: 6

# [tuneNumResyncSlaves]
# The number of threads (per target) used to perform the actual file and
# directory synchronizations for a buddy mirror resync.
# Default: 12

# [tuneNumStreamListeners]
# The number of threads waiting for incoming data events. Connections with
# incoming data will be handed over to the worker threads for actual message
# processing.
# Default: 1

# [tuneNumWorkers]
# The number of worker threads. Higher number of workers allows the server to
# handle more client requests in parallel, which also results in more
# concurrent access to the underlying storage device.
# Note: See also tuneUsePerTargetWorkers.
# Default: 12

# [tuneUseAggressiveStreamPoll]
# If set to true, the StreamListener component, which waits for incoming
# requests, will keep actively polling for events instead of sleeping until
# an event occurs. Active polling will reduce latency for processing of
# incoming requests at the cost of higher CPU usage.
# Default: false

# [tuneUsePerTargetWorkers]
# If set to true, a separate set of worker threads is created and exclusively
# assigned to each attached storage target. If set to false, a global set of
# worker threads is used and each worker thread can handle requests for all
# targets.
# Separate worker threads are intended to improve balance of I/O workload
# across targets under high load (i.e. when the number of concurrently incoming
# requests is higher than the number of worker threads).
# Note: If set to true, the actual number of created worker threads is
#    tuneNumWorkers x number_of_attached_targets.
# Default: true

# [tuneUsePerUserMsgQueues]
# If set to true, per-user queues will be used to decide which of the pending
# requests is handled by the next available worker thread. If set to false, a
# single queue will be used and incoming requests will be processed in
# first-come, first-served order.
# Per-user queues are intended to improve fairness in multi-user environments.
# Default: false

# [tuneWorkerBufSize]
# The buffer size, which is allocated twice by each worker thread for IO and
# network data buffering.
# Note: For optimal performance, this value must be at least 1MB higher than
#    tuneFileReadSize and tuneFileWriteSize.
# Default: 4m
